{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Ch `07`: Concept `01`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "All we'll need is TensorFlow and NumPy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Instead of feeding all the training data to the training op, we will feed data in small batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_batch(X, size):\n",
    "    a = np.random.choice(len(X), size, replace=False)\n",
    "    return X[a]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define the autoencoder class:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Autoencoder:\n",
    "    def __init__(self, input_dim, hidden_dim, epoch=500, batch_size=10, learning_rate=0.001):\n",
    "        self.epoch = epoch\n",
    "        self.batch_size = batch_size\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "        # Define input placeholder\n",
    "        x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim])\n",
    "        \n",
    "        # Define variables\n",
    "        with tf.name_scope('encode'):\n",
    "            weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights')\n",
    "            biases = tf.Variable(tf.zeros([hidden_dim]), name='biases')\n",
    "            encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases)\n",
    "        with tf.name_scope('decode'):\n",
    "            weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights')\n",
    "            biases = tf.Variable(tf.zeros([input_dim]), name='biases')\n",
    "            decoded = tf.matmul(encoded, weights) + biases\n",
    "\n",
    "        self.x = x\n",
    "        self.encoded = encoded\n",
    "        self.decoded = decoded\n",
    "\n",
    "        # Define cost function and training op\n",
    "        self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded))))\n",
    "\n",
    "        self.all_loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded)), 1))\n",
    "        self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n",
    "        \n",
    "        # Define a saver op\n",
    "        self.saver = tf.train.Saver()\n",
    "\n",
    "    def train(self, data):\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for i in range(self.epoch):\n",
    "                for j in range(500):\n",
    "                    batch_data = get_batch(data, self.batch_size)\n",
    "                    l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data})\n",
    "                if i % 50 == 0:\n",
    "                    print('epoch {0}: loss = {1}'.format(i, l))\n",
    "                    self.saver.save(sess, './model.ckpt')\n",
    "            self.saver.save(sess, './model.ckpt')\n",
    "        \n",
    "    def test(self, data):\n",
    "        with tf.Session() as sess:\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data})\n",
    "        print('input', data)\n",
    "        print('compressed', hidden)\n",
    "        print('reconstructed', reconstructed)\n",
    "        return reconstructed\n",
    "\n",
    "    def get_params(self):\n",
    "        with tf.Session() as sess:\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            weights, biases = sess.run([self.weights1, self.biases1])\n",
    "        return weights, biases\n",
    "\n",
    "    def classify(self, data, labels):\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data})\n",
    "            reconstructed = reconstructed[0]\n",
    "            # loss = sess.run(self.all_loss, feed_dict={self.x: data})\n",
    "            print('data', np.shape(data))\n",
    "            print('reconstructed', np.shape(reconstructed))\n",
    "            loss = np.sqrt(np.mean(np.square(data - reconstructed), axis=1))\n",
    "            print('loss', np.shape(loss))\n",
    "            horse_indices = np.where(labels == 7)[0]\n",
    "            not_horse_indices = np.where(labels != 7)[0]\n",
    "            horse_loss = np.mean(loss[horse_indices])\n",
    "            not_horse_loss = np.mean(loss[not_horse_indices])\n",
    "            print('horse', horse_loss)\n",
    "            print('not horse', not_horse_loss)\n",
    "            return hidden[7,:]\n",
    "\n",
    "    def decode(self, encoding):\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            reconstructed = sess.run(self.decoded, feed_dict={self.encoded: encoding})\n",
    "        img = np.reshape(reconstructed, (32, 32))\n",
    "        return img\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The *Iris dataset* is often used as a simple training dataset to check whether a classification algorithm is working. The sklearn library comes with it, `pip install sklearn`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0: loss = 3.8637373447418213\n",
      "epoch 50: loss = 0.25829368829727173\n",
      "epoch 100: loss = 0.3230888843536377\n",
      "epoch 150: loss = 0.3295430839061737\n",
      "epoch 200: loss = 0.24636892974376678\n",
      "epoch 250: loss = 0.22375555336475372\n",
      "epoch 300: loss = 0.19688692688941956\n",
      "epoch 350: loss = 0.2520211935043335\n",
      "epoch 400: loss = 0.29669439792633057\n",
      "epoch 450: loss = 0.2794385552406311\n",
      "input [[8, 4, 6, 2]]\n",
      "compressed [[ 0.72223264]]\n",
      "reconstructed [[ 6.87640762  2.79334426  6.23228502  2.21386957]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 6.87640762,  2.79334426,  6.23228502,  2.21386957]], dtype=float32)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "hidden_dim = 1\n",
    "data = datasets.load_iris().data\n",
    "input_dim = len(data[0])\n",
    "ae = Autoencoder(input_dim, hidden_dim)\n",
    "ae.train(data)\n",
    "ae.test([[8, 4, 6, 2]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
